<?php
/**
 * Created by PhpStorm.
 * User: lih
 * Date: 2017/11/7
 * Time: 22:26
 */

namespace common\components;

require_once(dirname(__FILE__).'/../libs/simplehtmldom/simple_html_dom.php');

use Yii;
use common\components\PostMan;

class CrawlHelper
{
    public $mRule;

    public function useRule($rule)
    {
        $this->mRule = $rule;
    }

    public function loadList($url)
    {
        $result = array();
        try {
            $rule = $this->mRule;
            $postman = new PostMan();
            $htmlCnt = $postman->httpGet($url);
            $html = str_get_html($htmlCnt);
            $listDom = $this->findElementsByXpath($html, $rule['itemQuery']);
            if(!empty($listDom)) {
                $listResult = array();
                $attrList = $rule['itemArgs'];
                foreach($listDom as $item) {
                    $itemResult = array();
                    foreach($attrList as $attr) {
                        $val = $this->findAttrByXpath($item, $attr['query'], $url);
                        if($val !== null) {
                            $itemResult[$attr['name']] = $val;
                        }
                    }
                    if(!empty($rule['itemDetailQuery'])) {
                        $val = $this->findAttrByXpath($item, $rule['itemDetailQuery'], $url);
                        if($val !== null) {
                            $itemResult['_detailUrl'] = $val;
                        }
                    }
                    $listResult[] = $itemResult;
                }
                $result = $listResult;
            }
            $html->clear();
            $htmlCnt = null;

        } catch(\Exception $e) {
            Yii::info("CrawlHelper exception: " . $e->getMessage());
        }
        return $result;
    }

    public function loadDetail($url)
    {
        $result = array();
        try {
            $rule = $this->mRule;
            $postman = new PostMan();
            $htmlCnt = $postman->httpGet($url);
            $html = str_get_html($htmlCnt);
            $detailDom = $this->findElementsByXpath($html, $rule['detailQuery']);
            if(!empty($detailDom)) {
                $attrList = $rule['detailArgs'];
                $itemResult = array();
                foreach ($attrList as $attr) {
                    if (!empty($attr['isArray'])
                            && !empty($attr['subQuery'])) {
                        $detailListDom = $this->findElementsByXpath($detailDom, $attr['query']);
                        if (!empty($detailListDom)) {
                            $subItemArray = array();
                            foreach ($detailListDom as $subItem) {
                                $val = $this->findAttrByXpath($subItem, $attr['subQuery'], $url);
                                if (!empty($val)) {
                                    $subItemArray[] = $val;
                                }
                            }
                            $itemResult[$attr['name']] = $subItemArray;
                        }
                    } else {
                        $val = $this->findAttrByXpath($detailDom, $attr['query'], $url);
                        if (!empty($val)) {
                            $itemResult[$attr['name']] = $val;
                        }
                    }
                }
                $result['detail'] = $itemResult;

                if (!empty($rule['detailNpQuery'])) {
                    $npurl = $this->findAttrByXpath($html, $rule['detailNpQuery'], $url);
                    if(!empty($npurl)) {
                        if(!empty($rule['detailNpRepPost'])) {
                            $nprep = $rule['detailNpRepPost'];
                            $matches = array();
                            if(preg_match("/{$nprep}/i", $npurl, $matches)) {
                                $npurl = $matches[1];
                                $result['npurl'] = $npurl;
                            }
                        } else {
                            $result['npurl'] = $npurl;
                        }
                    }
                }
            }
            $html->clear();
            $htmlCnt = null;

        } catch(\Exception $e) {
            Yii::info("CrawlHelper exception: " . $e->getMessage());
        }
        return $result;
    }

    public function loadTest($rule)
    {
        $result = array();
        try {
            if ($rule !== null) {
                $url = $rule['listUrl'];
                $html = file_get_html($url);
                $listDom = $this->findElementsByXpath($html, $rule['itemQuery']);
                if(!empty($listDom)) {
                    $listResult = array();
                    $attrList = $rule['itemArgs'];
                    foreach($listDom as $item) {
                        $itemResult = array();
                        foreach($attrList as $attr) {
                            $val = $this->findAttrByXpath($item, $attr['query'], $url);
                            if($val !== null) {
                                $itemResult[$attr['name']] = $val;
                            }
                        }
                        if(!empty($rule['itemDetailQuery'])) {
                            $val = $this->findAttrByXpath($item, $rule['itemDetailQuery'], $url);
                            if($val !== null) {
                                $itemResult['_detailUrl'] = $val;
                            }
                        }
                        $listResult[] = $itemResult;
                    }
                    $result['list'] = $listResult;
                }
                $html->clear();

                $url = '';
                if(!empty($result['list'])) {
                    $itemFirst = $result['list'][0];
                    if(!empty($itemFirst['_detailUrl'])) {
                        $url = $itemFirst['_detailUrl'];
                    }
                }
                if(!empty($url)) {
                    $html = file_get_html($url);
                    $detailDom = $this->findElementsByXpath($html, $rule['detailQuery']);
                    if(!empty($detailDom)) {
                        $attrList = $rule['detailArgs'];
                        $itemResult = array();
                        foreach($attrList as $attr) {
                            if (!empty($attr['isArray'])
                                    && !empty($attr['subQuery'])) {
                                $detailListDom = $this->findElementsByXpath($detailDom, $attr['query']);
                                if (!empty($detailListDom)) {
                                    $subItemArray = array();
                                    foreach ($detailListDom as $subItem) {
                                        $val = $this->findAttrByXpath($subItem, $attr['subQuery'], $url);
                                        if ($val !== null) {
                                            $subItemArray[] = $val;
                                        }
                                    }
                                    $itemResult[$attr['name']] = $subItemArray;
                                }
                            } else {
                                $val = $this->findAttrByXpath($detailDom, $attr['query'], $url);
                                if ($val !== null) {
                                    $itemResult[$attr['name']] = $val;
                                }
                            }
                        }
                        $result['detail'] = $itemResult;

                        if(!empty($rule['detailNpQuery'])) {
                            $result['detailNp'] = $this->findAttrByXpath($html, $rule['detailNpQuery'], $url);
                        }
                    }
                    $html->clear();
                }
            }
        } catch(\Exception $e) {
            Yii::info("CrawlHelper exception: " . $e->getMessage());
        }
        return $result;
    }

    /**
     * @param $dom \simple_html_dom
     * @param $xpath string
     * @return null|\simple_html_dom
     */
    private function findElementsByXpath($dom, $xpath)
    {
        $result = null;
        if(is_bool($dom)) {
            return $result;
        }
        try {
            $ruleList = explode('@', $xpath);
            if (!empty($ruleList)) {
                for ($i = 0; $i < count($ruleList); $i += 2) {
                    $rule = $ruleList[$i];
                    $ruleNext = null;
                    if($i + 1 < count($ruleList)) {
                        $ruleNext = $ruleList[$i + 1];
                    }
                    if($rule != null && strlen($rule) > 0) {
                        if($ruleNext !== null) {
                            $dom = $dom->find($rule, intval($ruleNext));
                            if(is_bool($dom)) {
                                $result = null;
                                break;
                            }
                            $result = $dom;
                        } else {
                            $dom = $dom->find($rule);
                            if(is_bool($dom)) {
                                $result = null;
                                break;
                            }
                            $result = $dom;
                            break;
                        }
                    }
                    if(empty($dom)) {
                        break;
                    }
                }
            }
        } catch(\Exception $e) {
        }
        return $result;
    }

    /**
     * @param  $dom
     * @param string $xpath
     */
    /**
     * @param $dom \simple_html_dom
     * @param $xpath $string
     * @param $urlTarget $string
     * @return null|array|\simple_html_dom
     */
    private function findAttrByXpath($dom, $xpath, $urlTarget)
    {
        $result = null;
        try {
            $ruleList = explode('@', $xpath);
            if (!empty($ruleList)) {
                for ($i = 0; $i < count($ruleList); $i += 2) {
                    $rule = $ruleList[$i];
                    $ruleNext = null;
                    if($i + 1 < count($ruleList)) {
                        $ruleNext = $ruleList[$i + 1];
                    }
                    if($rule != null && strlen($rule) > 0) {
                        if($ruleNext !== null) {
                            $dom = $dom->find($rule, intval($ruleNext));
                        } else {
                            if(0 === strpos($rule, 'abs:')) {
                                $rule = substr($rule, strlen('abs:'));
                                $result = $dom->getAttribute($rule);
                                if(!empty($result)) {
                                    $result = $this->fillAbsPath($urlTarget, $result);
                                }
                            } else {
                                $result = $dom->getAttribute($rule);
                            }
                            break;
                        }
                    } else {
                        $result = $dom->getAttribute('plaintext');
                        break;
                    }
                    if(empty($dom)) {
                        break;
                    }
                }
            }

        } catch(\Exception $e) {
        }
        return $result;
    }

    private function fillAbsPath($urlTarget, $linkChild) {
        if(strpos($linkChild, '://') > 0) {
            return $linkChild;
        }
        if(0 === strpos($linkChild, '/')) {
            $urlParse = parse_url($urlTarget);
            return $urlParse['scheme'] . '://' . $urlParse['host'] . $linkChild;
        }
        $urlPath = $urlTarget;
        if(strrpos($urlPath, '/') > 0) {
            $urlPath = substr($urlPath, 0, strrpos($urlPath, '/'));
        }
        return $urlPath . '/' . $linkChild;
    }
}